{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "235ce22dcabb4c00a14f13b690090e78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5811741a43354938976769614c06d784",
              "IPY_MODEL_8473657274ce4e818e3a50cdc1a305aa",
              "IPY_MODEL_7fc2445a4fa946b28308a54c4b9f2a0c"
            ],
            "layout": "IPY_MODEL_8e01b3b8249247e6a119c0ef033dcdc2"
          }
        },
        "5811741a43354938976769614c06d784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b7bf7d0d68a4f5ca6c4d9eb560d6ca3",
            "placeholder": "​",
            "style": "IPY_MODEL_3b8e58445ba14e208cdd752fcd71a9cd",
            "value": "Map: 100%"
          }
        },
        "8473657274ce4e818e3a50cdc1a305aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c5666484a7f48ffa947c03cd5d48247",
            "max": 872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3a5fe7df33da42c39df2bcfeb3ce8670",
            "value": 872
          }
        },
        "7fc2445a4fa946b28308a54c4b9f2a0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65a5418bc2e148e78344eb143ce49e74",
            "placeholder": "​",
            "style": "IPY_MODEL_38a871e60c2c40bb8a6e97750da139f8",
            "value": " 872/872 [00:00&lt;00:00, 2967.40 examples/s]"
          }
        },
        "8e01b3b8249247e6a119c0ef033dcdc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b7bf7d0d68a4f5ca6c4d9eb560d6ca3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b8e58445ba14e208cdd752fcd71a9cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c5666484a7f48ffa947c03cd5d48247": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a5fe7df33da42c39df2bcfeb3ce8670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "65a5418bc2e148e78344eb143ce49e74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38a871e60c2c40bb8a6e97750da139f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BSbU-4DFCAcY",
        "outputId": "4ee001f3-a40a-48a3-dffe-6ada207ffe95"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m50.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install -U pyarrow --quiet"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets transformers torch seqeval evaluate  --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GUkDF8leCxIF",
        "outputId": "cde17754-0b82-429d-db77-3fb6778d0a02"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/43.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m527.3/527.3 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import numpy as np\n",
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments\n",
        "from datasets import load_dataset\n",
        "from typing import Dict, List\n",
        "\n",
        "class EWC(nn.Module):\n",
        "    def __init__(self, model: nn.Module, dataset, tokenizer, importance: float = 1000):\n",
        "        super().__init__()\n",
        "        self.model = model\n",
        "        self.dataset = dataset\n",
        "        self.tokenizer = tokenizer\n",
        "        self.importance = importance\n",
        "        self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad}\n",
        "        self._means = {}\n",
        "        self._precision_matrices = {}\n",
        "        self.device = next(model.parameters()).device\n",
        "        self._initialize_means_and_precision_matrices()\n",
        "        self._calculate_importance()\n",
        "\n",
        "    def _initialize_means_and_precision_matrices(self):\n",
        "        for n, p in tqdm(self.params.items()):\n",
        "            self._means[n] = p.clone().detach().to(self.device)\n",
        "            self._precision_matrices[n] = p.clone().detach().fill_(0).to(self.device)\n",
        "\n",
        "    def _calculate_importance(self):\n",
        "        self.model.eval()\n",
        "        for i in tqdm(range(len(self.dataset))):\n",
        "            self.model.zero_grad()\n",
        "            data = self.dataset[i]\n",
        "            inputs = self.tokenizer(data['sentence'], return_tensors='pt', padding=True, truncation=True)\n",
        "            inputs = {k: v.to(self.device) for k, v in inputs.items()}\n",
        "            labels = torch.tensor([data['label']]).to(self.device)\n",
        "            outputs = self.model(**inputs, labels=labels)\n",
        "            loss = outputs.loss\n",
        "            loss.backward()\n",
        "\n",
        "            for n, p in tqdm(self.model.named_parameters()):\n",
        "                self._precision_matrices[n].data += p.grad.data ** 2 / len(self.dataset)\n",
        "\n",
        "    def penalty(self):\n",
        "        loss = 0\n",
        "        for n, p in self.model.named_parameters():\n",
        "            _loss = self._precision_matrices[n] * (p.to(self.device) - self._means[n]) ** 2\n",
        "            loss += _loss.sum()\n",
        "        return self.importance * loss\n",
        "\n",
        "    def update(self):\n",
        "        for n, p in tqdm(self.model.named_parameters()):\n",
        "            self._means[n] = p.clone().detach().to(self.device)\n",
        "\n",
        "class EWCTrainer(Trainer):\n",
        "    def __init__(self, ewc, *args, **kwargs):\n",
        "        super().__init__(*args, **kwargs)\n",
        "        self.ewc = ewc\n",
        "\n",
        "    def compute_loss(self, model, inputs, return_outputs=False):\n",
        "        outputs = model(**inputs)\n",
        "        loss = outputs.loss\n",
        "        ewc_loss = self.ewc.penalty()\n",
        "        total_loss = loss + ewc_loss\n",
        "        return (total_loss, outputs) if return_outputs else total_loss\n",
        "\n",
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return {\"accuracy\": (predictions == labels).mean()}\n",
        "\n",
        "def main():\n",
        "    # Load dataset\n",
        "    dataset = load_dataset(\"glue\", \"sst2\")\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "    def tokenize_function(examples):\n",
        "        return tokenizer(examples[\"sentence\"], padding=\"max_length\", truncation=True)\n",
        "\n",
        "    tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
        "    tokenized_datasets = tokenized_datasets.remove_columns(['sentence', 'idx'])\n",
        "    tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
        "    tokenized_datasets.set_format(\"torch\")\n",
        "\n",
        "    # Load pre-trained model\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    model = AutoModelForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=2).to(device)\n",
        "\n",
        "    # Initialize EWC\n",
        "    ewc = EWC(model, dataset[\"train\"].select(range(100)), tokenizer)  # Use a subset for EWC initialization\n",
        "\n",
        "    # Training arguments\n",
        "    training_args = TrainingArguments(\n",
        "        output_dir=\"./results\",\n",
        "        num_train_epochs=3,\n",
        "        per_device_train_batch_size=8,\n",
        "        per_device_eval_batch_size=8,\n",
        "        warmup_steps=500,\n",
        "        weight_decay=0.01,\n",
        "        logging_dir=\"./logs\",\n",
        "    )\n",
        "\n",
        "    # Initialize EWCTrainer\n",
        "    trainer = EWCTrainer(\n",
        "        ewc=ewc,\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        train_dataset=tokenized_datasets[\"train\"],\n",
        "        eval_dataset=tokenized_datasets[\"validation\"],\n",
        "        compute_metrics=compute_metrics,\n",
        "    )\n",
        "\n",
        "    # Evaluate the model\n",
        "    eval_results = trainer.evaluate()\n",
        "    print(eval_results)\n",
        "\n",
        "    # Train the model\n",
        "    trainer.train()\n",
        "\n",
        "    # Update EWC means after training\n",
        "    ewc.update()\n",
        "\n",
        "    # Evaluate the model\n",
        "    eval_results = trainer.evaluate()\n",
        "    print(eval_results)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "235ce22dcabb4c00a14f13b690090e78",
            "5811741a43354938976769614c06d784",
            "8473657274ce4e818e3a50cdc1a305aa",
            "7fc2445a4fa946b28308a54c4b9f2a0c",
            "8e01b3b8249247e6a119c0ef033dcdc2",
            "0b7bf7d0d68a4f5ca6c4d9eb560d6ca3",
            "3b8e58445ba14e208cdd752fcd71a9cd",
            "0c5666484a7f48ffa947c03cd5d48247",
            "3a5fe7df33da42c39df2bcfeb3ce8670",
            "65a5418bc2e148e78344eb143ce49e74",
            "38a871e60c2c40bb8a6e97750da139f8"
          ]
        },
        "id": "naG0gsQjCA7c",
        "outputId": "8c3a9718-089a-4f39-cff3-5816806d7b4a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/872 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "235ce22dcabb4c00a14f13b690090e78"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='218' max='109' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [109/109 1:21:30]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.7311079502105713, 'eval_accuracy': 0.4908256880733945, 'eval_runtime': 6.3536, 'eval_samples_per_second': 137.246, 'eval_steps_per_second': 17.156}\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='25257' max='25257' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [25257/25257 1:21:17, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.463400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.403700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.370500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.368000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.355900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.337200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.348400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.316900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.331700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.326800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.335400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.331100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.316600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.310100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.333900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.286000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.266400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.235300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.218500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.240600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.226600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.212400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.218500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.218000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.220900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.231500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.208400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.223900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.207700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.209600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.213900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.193300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.211100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.172700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.131200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.141000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.144000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.141700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.128300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.145200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20500</td>\n",
              "      <td>0.139700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>21000</td>\n",
              "      <td>0.135800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>21500</td>\n",
              "      <td>0.134600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>22000</td>\n",
              "      <td>0.132800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>22500</td>\n",
              "      <td>0.148000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>23000</td>\n",
              "      <td>0.120500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>23500</td>\n",
              "      <td>0.126200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>24000</td>\n",
              "      <td>0.135900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>24500</td>\n",
              "      <td>0.120000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>25000</td>\n",
              "      <td>0.125500</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.3378593325614929, 'eval_accuracy': 0.9185779816513762, 'eval_runtime': 6.4433, 'eval_samples_per_second': 135.335, 'eval_steps_per_second': 16.917, 'epoch': 3.0}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JHr8iZ9NEW8O"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}